/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package cz.mzk.oai.metadataRepository.harvest;

import ORG.oclc.oai.harvester.io.OAIReader;
import ORG.oclc.oai.harvester.verb.OAIError;
import ORG.oclc.oai.harvester.verb.Record;
import cz.mzk.oai.metadataRepository.Tools;
import cz.mzk.oai.metadataRepository.metadata.DateStamp;
import cz.mzk.oai.metadataRepository.repository.MetadataBase;
import cz.mzk.oai.metadataRepository.repository.Repository;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.logging.FileHandler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import org.xml.sax.SAXException;

/**
 *
 * @author Martin
 */
public class OAIHarvester implements MarcHarvester {

    private final Repository repository;
    private final URL providerUrl;
    private final String setSpec;
    private final FileHandler logHandler;
    private static final String PRIMARY_FORMAT = "marc21";

    /**
     *
     * @param providerUrl oai-pmh provider url
     * @param setSpec name of harvested set or null if whole repository should be harvested
     * @param repository repository in which harvester puts harvested data
     * @param logFileName full name of log file
     * @throws IOException if the log file could not be intitialized
     */
    public OAIHarvester(URL providerUrl, String setSpec, Repository repository, String logFileName) throws IOException {
        this.providerUrl = providerUrl;
        this.setSpec = setSpec;
        this.repository = repository;
        Logger logger = Logger.getLogger(DirectoryHarvester.class.getName());
        logHandler = new FileHandler(logFileName, true);
        logHandler.setFormatter(new SimpleFormatter());
        logger.addHandler(logHandler);
        //System.out.println("creating OAIHarvester");
        //System.out.println("url: " + providerUrl.toString());
        //System.out.println("setSpec: " + setSpec);
    }

    /**
     * @deprecated
     */
    public static void harvestAleph(String destDir) {
        try {
            File rootDirFile = new File(destDir);
            if (!rootDirFile.exists() && rootDirFile.isDirectory()) {
                rootDirFile.mkdir();
                System.out.println("creating " + destDir);
            }
            URL aleph = new URL("http://aleph.mzk.cz/OAI");
            OAIReader reader = new OAIReader(aleph, null, null, "MZK03", PRIMARY_FORMAT);
            System.out.println("Harvesting ");
            Record record;
            String filename;
            File file;
            FileWriter writer;
            int counter = 0;
            while ((record = reader.readNext()) != null) {
                //System.out.println("id: " +  record.getIdentifier());
                filename = Tools.idToFileName(record.getIdentifier());
                //System.out.println("file: " + file.getAbsolutePath());
                try {
                    //System.out.println(record.getRecordXML().length());
                    Document doc = DocumentHelper.parseText(record.getRecordXML());
                    Element recordEl = doc.getRootElement();
                    Element metadataEl = recordEl.element("metadata");
                    if (metadataEl != null) {
                        Element marcRecord = metadataEl.element("record");
                        file = new File(destDir + File.separatorChar + filename);//record.getIdentifier() + ".xml");
                        writer = new FileWriter(file);
                        writer.write(marcRecord.asXML());
                        writer.close();
                    }
                    if (++counter % 1000 == 0) {
                        System.out.println("harvested " + counter + " records");
                    }
                } catch (DocumentException ex) {
                    Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
                }
                //break;
            }
        } catch (SAXException ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        } catch (MalformedURLException ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        } catch (OAIError ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    public void harvestAndReplace(DateStamp from, DateStamp until, boolean thorough) {
        harvest(from, until, true, thorough);
    }

    public void harvestAndUpdate(DateStamp from, DateStamp until, boolean thorough) {
        harvest(from, until, false, thorough);
    }

    /**
     *
     * @param from
     * @param until
     * @param forceReplace if true all harvested items will replace those in repository irrespective to their datestamp
     */
    private void harvest(DateStamp from, DateStamp until, boolean forceReplace, boolean thorough) {
        try {
            //TODO: pouzit harevester2 jakmile odpovi Jeff Young
            OAIReader reader = new OAIReader(this.providerUrl, from.toString(), until.toString(), setSpec, PRIMARY_FORMAT);
            System.out.println("Harvesting from " + from + " until " + until);
            Record record;
            int counter = 0;
            int added = 0;
            while ((record = reader.readNext()) != null) {
                //System.out.println("id: " +  record.getIdentifier());
                //System.out.println("file: " + file.getAbsolutePath());
                try {
                    //System.out.println(record.getRecordXML().length());
                    Document doc = DocumentHelper.parseText(record.getRecordXML());
                    Element recordEl = doc.getRootElement();
                    Element metadataEl = recordEl.element("metadata");
                    if (metadataEl != null) {
                        Element marcRecordEl = metadataEl.element("record");
                        Document marcRecordDoc = DocumentHelper.createDocument();
                        marcRecordDoc.add(marcRecordEl.detach());
                        String id = record.getIdentifier();
                        DateStamp datestamp;
                        if (forceReplace) {
                            datestamp = new DateStamp();
                        } else {
                            datestamp = new DateStamp(record.getDatestamp());
                        }
                        if (repository.addToCollection(MetadataBase.MZK03, marcRecordDoc, id, datestamp, thorough, false)) {
                            added++;
                            /* System.out.println("counter: " + counter);
                            System.out.println("datestamp: " + datestamp);
                            System.out.println("id: " + id);
                            //break;
                             */
                        }
                    }
                    if (++counter % 1000 == 0) {
                        System.out.println("harvested " + counter + " records");
                        System.out.println("added/replaced " + added + " records");
                    }
                } catch (DocumentException ex) {
                    Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
                }
            }
            System.out.println("harvested " + counter + " records");
            System.out.println("added/replaced " + added + " records");
        } catch (SAXException ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        } catch (MalformedURLException ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        } catch (OAIError ex) {
            Logger.getLogger(OAIHarvester.class.getName()).log(Level.SEVERE, null, ex);
        } finally {
            repository.updateProperties();
        }
    }

    public void close() {
        logHandler.close();
        repository.close();
    }
}
